#include "../include/KeywordProcessor.h"
#include "../include/DirectoryScanner.h"
#include "../include/log4cppuse.h"

#include <fstream>   
#include <iostream>  


#include <regex>


KeyWordProcessor::KeyWordProcessor()
:m_enStopWords(std::make_unique<std::set<std::string>>())
,m_chStopWords(std::make_unique<std::set<std::string>>())
,m_enWords(std::make_shared<std::map<std::string, int>>())
,m_chWords(std::make_shared<std::map<std::string, int>>())
,m_en_characters_index(std::make_shared<std::map<std::string, std::vector<int>>>())
,m_ch_characters_index(std::make_shared<std::map<std::string, std::vector<int>>>())
{
   
}


// chDir: 中文语料库
// enDir: 英文语料库
void KeyWordProcessor::process(const  std::string& chDir, 
                 const  std::string& enDir, 
                 const  std::string& cnDictOutfile,
                 const  std::string& cnIndexOutfile,
                 const  std::string& enDictOutfile,
                 const  std::string& enIndexOutfile)
{
    create_cn_dict(chDir, cn_dict_outfile);
    build_cn_index(cnIndexOutfile);
    create_en_dict(enDir, enDictOutfile);
    build_en_index(enIndexOutfile);
    m_enStopWords->clear();
    m_chStopWords->clear();
    m_enWords->clear();
    m_chWords->clear();
    m_en_characters_index->clear();
    m_ch_characters_index->clear();
    return ;
}

//进行将中文词典写入文件
void KeyWordProcessor::create_cn_dict(const std::string& dir, const std::string& outfile)
{
    load_ch_stop_words();
    std::vector<std::string> chFiles=DirectoryScanner::getDirectoryScanner().scan(dir);
    
     // 正则表达式：匹配符号和单个数字
    std::regex pattern(R"([!@#$%^&*()_+\-=\[\]{};':"\\|,.<>/?]|\b\d+\b)");
    

    for(auto fileName:chFiles)
    {
        // std::string filePath=dir+"/"+fileName;
        std::string filePath=fileName;
        std::ifstream infile(filePath);
        if (!infile.is_open()) {
            std::string log=filePath+":无法打开文件进行读取";
            LOG_ERROR(log.c_str());
            return;
        }

        std::string line;
        while (std::getline(infile,line))
        {
           
            // 步骤1: 将符号和单个数字替换为空格
            std::string processed_line = std::regex_replace(line, pattern, " ");
            
             // 步骤2: 分词
            std::vector<std::string> words;
            m_tokenizer.Cut(processed_line,words);
           
            // 步骤3: 过滤和统计
            for(auto& word:words)
            {
                
                // 跳过空字符串
                if (word.empty()) {
                    continue;
                }
                
                // 跳过停用词
                if (m_chStopWords->find(word) != m_chStopWords->end()) {
                    continue;
                }
                
                // 跳过单个字符（可能是残留的符号或数字）
                if (word.length() == 1 && !std::isalpha(word[0])) {
                    continue;
                }

                else{
                    auto const &w=utf8::iterator(word.begin(),word.begin(),word.end());
                    char32_t c=*w;
                    if((c>=0x3400 && c<=0x4DBF) || (c>=0x4E00 && c<=0x9FFF))
                    {
                        (*m_chWords)[word]++;
                    }
                    
                }
            }

        }
       infile.close(); 
    
    }

    //将词频写入文件
    std::ofstream output_file(outfile,std::ios::out|std::ios::trunc);
    if (!output_file.is_open()) {
        std::string log= outfile+":无法打开文件进行写入";
        LOG_ERROR(log.c_str());
        return;
    }
    for(auto& word:*m_chWords)
    {
        output_file<<word.first<<" "<<word.second<<std::endl;
    }
    output_file.close();

    std::string log="create_cn_dict success";
    LOG_INFO(log.c_str());

}

//进行将中文索引写入文件
void KeyWordProcessor::build_cn_index(const std::string& index)
{
    int i=0;
    for(auto& [word,_]:*m_chWords)
    {
        
         const char *it=word.c_str();
         const char *end=it+word.size();

         while(it!=end)
         {
            auto start=it;
            utf8::next(it,end);
            std::string character=std::string(start,it);
            (*m_ch_characters_index)[character].push_back(i);
         }

        i++;
    }

    //将索引写入文件
    std::ofstream output_file(index,std::ios::out|std::ios::trunc);
    if (!output_file.is_open()) {
        std::string log= index+":无法打开文件进行写入";
        LOG_ERROR(log.c_str());
        return;
    }
    for(auto& [character,indexs]:*m_ch_characters_index)
    {
        output_file<<character<<" ";
        for(auto& index:indexs)
        {
            output_file<<index<<" ";
        }
        output_file<<std::endl;
    }
    output_file.close();
    std::string log="build_cn_index success";
    LOG_INFO(log.c_str());
}
//进行将英文词典写入文件
void KeyWordProcessor::create_en_dict(const std::string& dir, const std::string& outfile)
{
    load_en_stop_words();
    std::vector<std::string> enFiles=DirectoryScanner::getDirectoryScanner().scan(dir);

    // 正则表达式：匹配符号和单个数字
    std::regex non_alpha_pattern(R"([^a-zA-Z])");

    for(auto fileName:enFiles)
    {
        // std::string filePath=dir+"/"+fileName;
        std::string filePath=fileName;
        std::ifstream infile(filePath);
        if (!infile.is_open()) {
            std::string log=filePath+":无法打开文件进行读取";
            LOG_ERROR(log.c_str());
            return;
        }

        std::string line;
        while (std::getline(infile,line))
        {
           
            // 跳过空行
            if (line.empty()) {
                continue;
            }
            // 步骤1: 将符号和单个数字替换为空格
           std::string processed_line = std::regex_replace(line, non_alpha_pattern, " ");
            
             // 去除多余空格（将多个连续空格替换为单个空格）
            processed_line = std::regex_replace(processed_line, std::regex("\\s+"), " ");

            // 去除首尾空格
            processed_line = std::regex_replace(processed_line, std::regex("^\\s+|\\s+$"), "");

            // 如果处理后为空，则跳过
            if (processed_line.empty()) {
                continue;
            }

             // 步骤2: 分词
            std::vector<std::string> words;
            words=split_by_space(processed_line);
           
            // 步骤3: 过滤和统计
            for(auto& word:words)
            {
                
                // 跳过空字符串
                if (word.empty()) {
                    continue;
                }
                // 跳过停用词
                if (m_enStopWords->find(word) != m_enStopWords->end()) {
                    continue;
                }
                
                // 跳过单个字符（可能是残留的符号或数字）
                if (word.length() == 1 && !std::isalpha(word[0])) {
                    continue;
                }

                else{
                    word=to_lower_case(word);
                    (*m_enWords)[word]++;
                }
            }

        }
       infile.close(); 

    }

    //将词频写入文件
    std::ofstream output_file(outfile,std::ios::out|std::ios::trunc);
    if (!output_file.is_open()) {
        std::string log= outfile+":无法打开文件进行写入";
        LOG_ERROR(log.c_str());
        return;
    }
    for(auto& word:*m_enWords)
    {
        output_file<<word.first<<" "<<word.second<<std::endl;
    }
    output_file.close();
    std::string log="create_en_dict success";
    LOG_INFO(log.c_str());
}


//进行将英文索引写入文件
void KeyWordProcessor::build_en_index(const std::string& index)
{
    int i=0;
    for(auto& [word,_]:*m_enWords)
    {
        
        // 遍历单词中的每个字符
        for (size_t j = 0; j < word.length(); ++j) {
            // 将字符转换为字符串
            std::string character = std::string(1, word[j]);
            
            // 将单词索引添加到该字符的索引列表中
            (*m_en_characters_index)[character].push_back(i);
        }

        i++;
    }
    //将索引写入文件
    std::ofstream output_file(index,std::ios::out|std::ios::trunc);
    if (!output_file.is_open()) {
        std::string log= index+":无法打开文件进行写入";
        LOG_ERROR(log.c_str());
        return;
    }
    for(auto& [character,indexs]:*m_en_characters_index)
    {
        output_file<<character<<" ";
        for(auto& index:indexs)
        {
            output_file<<index<<" ";
        }
        output_file<<std::endl;
    }
    output_file.close();
    std::string log="build_en_index success";
    LOG_INFO(log.c_str());
}



void KeyWordProcessor::load_en_stop_words(const std::string& filename)
{
    std::ifstream outfile(filename);
    if (!outfile.is_open()) {
        std::string log= filename+":无法打开文件进行读取";
        LOG_ERROR(log.c_str());
        return;
    }

    std::string work;
    while (std::getline(outfile,work))
    {
        m_enStopWords->insert(std::move(work));
    }
}

void KeyWordProcessor::load_ch_stop_words(const std::string& filename)
{
    std::ifstream outfile(filename);
    if (!outfile.is_open()) {
        std::string log= filename+":无法打开文件进行读取";
        LOG_ERROR(log.c_str());
        return;
    }

    std::string work;
    while (std::getline(outfile,work))
    {
        m_chStopWords->insert(std::move(work));
    }
}

// 使用空格切割文本
std::vector<std::string> KeyWordProcessor::split_by_space(const std::string& text) {
    std::vector<std::string> words;
    std::istringstream iss(text);
    std::string word;
    
    while (iss >> word) {
        words.push_back(word);
    }
    
    return words;
}
   // 转换为小写
std::string KeyWordProcessor::to_lower_case(const std::string& str) {
    std::string result = str;
    std::transform(result.begin(), result.end(), result.begin(), ::tolower);
    return result;
}

#if 0

int main()
{
    KeyWordProcessor keyWordProcessor;
  
    keyWordProcessor.process();
    return 0;
}
#endif